Library version 1.0 tests

In [157]:
## This code cell will not be shown in the HTML version of this notebook
# imports from custom library
import sys
sys.path.append('../../')
from mlrefined_libraries import nonlinear_superlearn_library as nonlib
from mlrefined_libraries import unsupervised_library as unsuplib

# demos for this notebook
regress_plotter = nonlib.nonlinear_regression_demos
datapath = '../../mlrefined_datasets/nonlinear_superlearn_datasets/'
unsup_datapath = '../../mlrefined_datasets/unsuperlearn_datasets/'

# import autograd functionality to bulid function's properly for optimizers
import autograd.numpy as np

# this is needed to compensate for %matplotlib notebook's tendancy to blow up images when plotted inline
%matplotlib notebook
from matplotlib import rcParams
rcParams['figure.autolayout'] = True

%load_ext autoreload
%autoreload 2

The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload

Regression

In [23]:
# load data
csvname = datapath + 'universal_regression_samples_0.csv'
csvname = datapath + 'universal_regression_function.csv'
csvname = datapath + 'discrete_function.csv'

# csvname = datapath + 'noisy_sin_sample.csv'
data = np.loadtxt(csvname,delimiter = ',')
x = data[:-1,:]
y = data[-1:,:] 

# plot dataset
demo = regress_plotter.Visualizer(data)
demo.plot_data()
In [25]:
x.shape
Out[25]:
(1, 1000)

Custom features

Put in a custom feature transformation - e.g., polynomial

In [46]:
# # a elliptical feature transformation
# def feature_transforms(x):
#     # calculate feature transform
#     f = []
#     for i in range(0,D):
#         for j in range(0,D-i):
#             if i > 0 or j > 0:
#                 term = (x[0,:]**i)*((x[1,:])**j)
#                 f.append(term)
#     return np.array(f)

def feature_transforms(x):
    # calculate feature transform
    f = np.array([(x.flatten()**d) for d in range(1,D)])   
    return f

def initializer():
    scale = 0.1
    r = scale*np.random.randn(D,1)
    return r
In [51]:
csvname = datapath + 'universal_regression_function.csv'
data = np.loadtxt(csvname,delimiter = ',')
x = data[:-1,:]
y = data[-1:,:] 

# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
D = 10
mylib.choose_features(name = 'custom',feature_transforms = feature_transforms,initializer = initializer)

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'least_squares')

# fit an optimization
mylib.fit(max_its = 3000,alpha_choice = 10**(-4))

# plot cost history
mylib.show_histories(start = 10)
In [ ]:
In [26]:
csvname = datapath + 'noisy_sin_sample.csv'
csvname = datapath + 'universal_regression_samples_0.csv'
csvname = datapath + 'universal_regression_function.csv'

Continuous function approximation

In [53]:
csvname = datapath + 'universal_regression_function.csv'
data = np.loadtxt(csvname,delimiter = ',')
x = data[:-1,:]
y = data[-1:,:] 

# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
mylib.choose_features(name = 'multilayer_perceptron',layer_sizes = [1,100,1],activation = 'tanh')

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'least_squares')

# fit an optimization
mylib.fit(max_its = 10000,alpha_choice = 10**(-2))

# plot cost history
mylib.show_histories(start = 10)
In [54]:
# load up animator
demo = nonlib.run_animators.Visualizer(csvname)

# pluck out a sample of the weight history
num_frames = 10 # how many evenly spaced weights from the history to animate

# animate based on the sample weight history
demo.animate_1d_regression(mylib,num_frames,scatter = 'none',show_history = True)
Out[54]:



Discrete function approximation

In [ ]:
csvname = datapath + 'discrete_function.csv'
data = np.loadtxt(csvname,delimiter = ',')
x = data[:-1,:]
y = data[-1:,:] 

# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
D = 10
# mylib.choose_features(name = 'custom',feature_transforms = feature_transforms,initializer = initializer)
mylib.choose_features(name = 'multilayer_perceptron',layer_sizes = [1,100,1],activation = 'tanh')

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'softmax')

# fit an optimization
mylib.fit(max_its = 10000,alpha_choice = 10**(-1))

# plot cost history
mylib.show_histories(start = 10)
In [41]:
# load up animator
demo = nonlib.run_animators.Visualizer(csvname)

# pluck out a sample of the weight history
num_frames = 100 # how many evenly spaced weights from the history to animate

# animate based on the sample weight history
demo.animate_1d_regression(mylib,num_frames,scatter = 'function',show_history = True)
Out[41]:



make and store a discrete function

In [55]:
# create instance of linear regression demo, used below and in the next examples
demo = nonlib.nonlinear_classification_visualizer.Visualizer(datapath + '2eggs_data.csv')
x = demo.x.T
y = demo.y[np.newaxis,:]

# an implementation of the least squares cost function for linear regression for N = 2 input dimension datasets
demo.plot_data()
In [137]:
# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
mylib.choose_features(name = 'multilayer_perceptron',layer_sizes = [2,100,1],activation = 'tanh')

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'least_squares')

# fit an optimization
mylib.fit(max_its = 10000,alpha_choice = 10**(-1))

# plot cost history
mylib.show_histories(start = 10)
In [138]:
demo = nonlib.run_animators.Visualizer(datapath + '2eggs_data.csv')
frames = 5
demo.animate_static_N2_simple(mylib,frames,show_history = False)
Out[138]:



In [141]:
# generate data
r1 = np.linspace(0,1,100)
r2 = np.linspace(0,1,100)
s,t = np.meshgrid(r1,r2)
s = np.reshape(s,(np.size(s),1))
t = np.reshape(t,(np.size(t),1))
h = np.concatenate((s,t),axis = 1)
w = mylib.weight_histories[0][-1]
z = mylib.model(mylib.normalizer(h.T),w)
z = np.sign(z)
In [144]:
data = np.vstack((h.T,z))
In [145]:
np.savetxt(datapath + '2eggs_function.csv',data,delimiter = ',')
In [146]:
# create instance of linear regression demo, used below and in the next examples
demo = nonlib.nonlinear_classification_visualizer.Visualizer(datapath + '2eggs_function.csv')
x = demo.x.T
y = demo.y[np.newaxis,:]

# an implementation of the least squares cost function for linear regression for N = 2 input dimension datasets
demo.plot_data()

normalizing architecture

In [272]:
# create initial weights for arbitrary feedforward network
def initialize_network_weights():
    # container for entire weight tensor
    weights = []
    
    # loop over desired layer sizes and create appropriately sized initial 
    # weight matrix for each layer
    for k in range(len(layer_sizes)-1):
        # get layer sizes for current weight matrix
        U_k = layer_sizes[k]
        U_k_plus_1 = layer_sizes[k+1]

        # make weight matrix
        weight = scale*np.random.randn(U_k+1,U_k_plus_1)
        if k < len(layer_sizes)-2:
            v = scale*np.random.randn(2,1)
            weights.append([weight,v])
        else:
            weights.append(weight)

    # re-express weights so that w_init[0] = omega_inner contains all 
    # internal weight matrices, and w_init = w contains weights of 
    # final linear combination in predict function
    w_init = [weights[:-1],weights[-1]]
    
    return w_init

# choose a nonlinear activation function 
def activation(t):
    nonlinearity = np.tanh(t)
    return nonlinearity

# fully evaluate our network features using the tensor of weights in w
def feature_transforms_normalized(a, w):    
    # loop through each layer matrix
    for W,v in w:
        #  pad with ones (to compactly take care of bias) for next layer computation        
        o = np.ones((1,np.shape(a)[1]))
        a = np.vstack((o,a))
        
        # compute inner product with current layer weights
        a = np.dot(a.T, W).T
    
        # output of layer activation
        a = activation(a)
        
        # normalize output
        mynorm = nonlib.library_v1.normalizers.Setup(a,name = 'standard')
        a = v[0]*mynorm.normalizer(a) + v[1]
    return a
In [269]:
layer_sizes = [1,3,5,1]
b = initialize_network_weights()
In [279]:
csvname = datapath + 'universal_regression_samples_0.csv'
data = np.loadtxt(csvname,delimiter = ',')
x = data[:-1,:]
y = data[-1:,:] 

layer_sizes = [1,10,10,1]; scale = 0.1

# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
mylib.choose_features(name = 'multilayer_perceptron',layer_sizes = layer_sizes,activation = 'tanh')

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'least_squares')

# fit an optimization
mylib.fit(max_its = 1000,alpha_choice = 10**(-2))

# choose features
mylib.choose_features(name = 'custom',feature_transforms = feature_transforms_normalized,initializer = initialize_network_weights)

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'least_squares')

# fit an optimization
mylib.fit(max_its = 1000,alpha_choice = 10**(-2))

# plot cost history
mylib.show_histories(start = 10,labels = ['run 1','run 2'])
In [280]:
# create instance of linear regression demo, used below and in the next examples
demo = nonlib.nonlinear_classification_visualizer.Visualizer(datapath + '2eggs_data.csv')
x = demo.x.T
y = demo.y[np.newaxis,:]

# an implementation of the least squares cost function for linear regression for N = 2 input dimension datasets
demo.plot_data()
In [288]:
layer_sizes = [2,10,10,10,1]; scale = 0.1

# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
mylib.choose_features(name = 'multilayer_perceptron',layer_sizes = layer_sizes,activation = 'tanh')

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'softmax')

# fit an optimization
mylib.fit(max_its = 1000,alpha_choice = 10**(-1))

# choose features
mylib.choose_features(name = 'custom',feature_transforms = feature_transforms_normalized,initializer = initialize_network_weights)

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'softmax')

# fit an optimization
mylib.fit(max_its = 1000,alpha_choice = 10**(-1))

# plot cost history
mylib.show_histories(start = 10,labels = ['run 1','run 2'])
In [ ]:
In [ ]:
In [306]:
# create initial weights for arbitrary feedforward network
def initialize_network_weights():
    # container for entire weight tensor
    weights = []
    
    # loop over desired layer sizes and create appropriately sized initial 
    # weight matrix for each layer
    for k in range(len(layer_sizes)-1):
        # get layer sizes for current weight matrix
        U_k = layer_sizes[k]
        U_k_plus_1 = layer_sizes[k+1]

        # make weight matrix
        weight = scale*np.random.randn(U_k+1,U_k_plus_1)
        weights.append(weight)

    # re-express weights so that w_init[0] = omega_inner contains all 
    # internal weight matrices, and w_init = w contains weights of 
    # final linear combination in predict function
    w_init = [weights[:-1],weights[-1]]
    
    return w_init

# choose a nonlinear activation function 
def activation(t):
    nonlinearity = np.tanh(t)
    return nonlinearity

# fully evaluate our network features using the tensor of weights in w
def feature_transforms_normalized(a, w):    
    # loop through each layer matrix
    for W in w:
        #  pad with ones (to compactly take care of bias) for next layer computation        
        o = np.ones((1,np.shape(a)[1]))
        a = np.vstack((o,a))
        
        # compute inner product with current layer weights
        a = np.dot(a.T, W).T
    
        # output of layer activation
        a = activation(a)
        
        # normalize output
        mynorm = nonlib.library_v1.normalizers.Setup(a,name = 'standard')
        a = mynorm.normalizer(a)
    return a
In [ ]:
In [ ]:
In [ ]:
In [300]:
# load in mnist test set
csvname = '/Users/Nurgetson/Desktop/mnist/mnist_test_contrast_normalized.csv'
data = np.loadtxt(csvname,delimiter = ',')

# get input/output pairs
x = data[:-1,:]
y = data[-1:,:]
In [295]:
x.shape
Out[295]:
(9999, 785)
In [301]:
data.shape
Out[301]:
(785, 10000)
In [305]:
# define layer size
layer_sizes = [784,10,10,10,10]; scale = 0.1

# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
mylib.choose_features(name = 'multilayer_perceptron',layer_sizes = layer_sizes,activation = 'tanh')

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'multiclass_softmax')

# fit an optimization
mylib.fit(max_its = 1000,alpha_choice = 10**(-1))

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-305-5affb4952a2b> in <module>()
     27 
     28 # fit an optimization
---> 29 mylib.fit(max_its = 1000,alpha_choice = 10**(-1))
     30 
     31 # plot cost history

/Users/Nurgetson/Dropbox/mlrefined/mlrefined_libraries/nonlinear_superlearn_library/library_v1/superlearn_setup.py in fit(self, **kwargs)
     71         # basic parameters for gradient descent run (default algorithm)
     72         max_its = 500; alpha_choice = 10**(-1);
---> 73         self.w_init = self.initializer()
     74 
     75         # set parameters by hand

TypeError: initialize_network_weights() missing 2 required positional arguments: 'layer_sizes' and 'scale'
In [307]:
# choose features
mylib.choose_features(name = 'custom',feature_transforms = feature_transforms_normalized,initializer = initialize_network_weights)

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'multiclass_softmax')

# fit an optimization
mylib.fit(max_its = 1000,alpha_choice = 10**(-1))

# plot cost history
mylib.show_histories(start = 10,labels = ['run 1','run 2'])
In [ ]:

backend trees

In [19]:
# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
D = 5
mylib.choose_features(name = 'stumps')

# choose normalizer
mylib.choose_normalizer(name = 'none')

# choose cost
mylib.choose_cost(name = 'least_squares')

# fit an optimization
mylib.fit(max_its = 1000,alpha_choice = 10**(-2))

# plot cost history
mylib.show_histories(start = 10)
In [20]:
# plot dataset
demo = regress_plotter.Visualizer(data)

# pluck out best weights - those that provided lowest cost, 
# and plot resulting fit
ind = mylib.cost_histories[-1]
w_best = mylib.weight_histories[0][-1]
demo.plot_fit(w_best,mylib.model,normalizer = mylib.normalizer);

single variable animation

In [ ]:
aninmation_demo = nonlib.run_animators.Visualizer(datapath + '2eggs_data.csv')
weight_history = run.weight_history[40:90]
aninmation_demo.animate_static_N2_simple(weight_history,run)
In [ ]:

two-class classification

In [6]:
# load data
csvname = datapath + 'signed_projectile.csv'
data = np.loadtxt(csvname,delimiter = ',')
x = data[:-1,:]
y = data[-1:,:]
In [7]:
# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
mylib.choose_features(name = 'multilayer_perceptron',layer_sizes = [1,1],activation = 'linear')

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'softmax')

# fit an optimization
mylib.fit(max_its = 100,alpha_choice = 10**(-1))

# plot histories
mylib.show_histories()
In [21]:
demo = nonlib.nonlinear_classification_visualizer.Visualizer(datapath + 'ellipse_2class_data.csv')
x = demo.x.T
y = demo.y[np.newaxis,:]
In [22]:
# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
mylib.choose_features(name = 'stumps')

# mylib.choose_features(name = 'multilayer_perceptron',layer_sizes = [2,10,10,10,1],activation = 'tanh')

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'softmax')

# fit an optimization
mylib.fit(max_its = 100,alpha_choice = 10**(0))

# plot histories
mylib.show_histories()

Multiclass classification

In [21]:
demo = nonlib.nonlinear_classification_visualizer.Visualizer(datapath + '3_layercake_data.csv')
x = demo.x.T
y = demo.y[:,np.newaxis]
In [29]:
# import the v1 library
mylib = nonlib.library_v1.superlearn_setup.Setup(x,y)

# choose features
# mylib.choose_features(name = 'multilayer_perceptron',layer_sizes = [2,10,10,3],activation = 'tanh')
mylib.choose_features(name = 'stumps',num_classifiers = 3)

# choose normalizer
mylib.choose_normalizer(name = 'none')

# choose cost
mylib.choose_cost(name = 'multiclass_softmax')

# fit an optimization
mylib.fit(max_its = 100,alpha_choice = 10**(-2))

# plot histories
mylib.show_histories()

Autoencoder

In [24]:
# import data
datapath = '../../mlrefined_datasets/nonlinear_superlearn_datasets/'
p = np.loadtxt(datapath + 'universal_autoencoder_manifold_0.csv',delimiter=',')
X = np.loadtxt(datapath + 'universal_autoencoder_samples_0.csv',delimiter=',')

# # scatter dataset
# fig = plt.figure(figsize = (9,4))
# gs = gridspec.GridSpec(1,1) 
# ax = plt.subplot(gs[0],aspect = 'equal'); 
# ax.set_xlabel(r'$x_1$',fontsize = 15);ax.set_ylabel(r'$x_2$',fontsize = 15,rotation = 0);
# ax.scatter(X[0,:],X[1,:],c = 'k',s = 60,linewidth = 0.75,edgecolor = 'w')
# plt.show()
In [25]:
mylib = nonlib.library_v1.unsuperlearn_setup.Setup(X)

# choose features
mylib.choose_encoder(layer_sizes = [2,10,1],scale = 0.2)
mylib.choose_decoder(layer_sizes = [1,10,2],scale = 0.2)

# choose normalizer
mylib.choose_normalizer(name = 'standard')

# choose cost
mylib.choose_cost(name = 'autoencoder')

# fit an optimization
mylib.fit(max_its = 1000,alpha_choice = 10**(-1))

# plot histories
mylib.show_histories()
In [26]:
# plot results
nonlib.autoencoder_demos.show_encode_decode(X,mylib,projmap = False,scale = 10)
In [ ]: